import json
import logging
import os.path
import re
import shutil
import sys
import time

import requests


import logging

from pymongo import MongoClient


def connect_to_mongodb():
    try:
        # 连接到本地MongoDB实例
        client = MongoClient('localhost', 27017)
        # 选择数据库
        db = client['douban_movies']
        # 选择集合
        collection = db['top250']
        return collection
    except Exception as e:
        logging.error(f"连接MongoDB失败: {e}")
        return None

# 在数据处理完成后存储到MongoDB
def save_to_mongodb(data):
    collection = connect_to_mongodb()
    if collection is not None:
        try:
            # 插入所有数据
            result = collection.insert_many(data)
            logging.info(f"成功插入 {len(result.inserted_ids)} 条记录到MongoDB")
            return True
        except Exception as e:
            logging.error(f"插入数据到MongoDB失败: {e}")
            return False
    return False


if os .path.exists("../static/DOuBai"):
    shutil.rmtree("../static/DOuBai")
os.makedirs("../static/DOuBai")

def logger_config():
    logging.basicConfig(
        level=logging.INFO,
        format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
        handlers=[
            logging.FileHandler('../static/DOuBai/log.txt', encoding='utf-8'),  # 日志文件
            logging.StreamHandler()  # 控制台输出
        ]
    )
logger_config()
logger = logging.getLogger('__name__')
if  os.path.exists("Img"):
    shutil.rmtree("Img")
os.makedirs("Img")
target_file = "Img"

header = {"user-agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/141.0.0.0 Safari/537.36 Edg/141.0.0.0"
}
logger.info(f"正在爬取中......")
all_datas = []
try:
    for page in range(0, 250, 25):
        url = f"https://movie.douban.com/top250?start={page}&filter=unwatched"

        response = requests.get(url, headers=header)

        items = re.findall(r'<img .*?alt="(.*?)" src="(.*?)">', response.text)

        for item in items:
            logger.info(f"正在爬取中......电影主题是：{item[0]}，封面图片是：{item[1]}")


            all_datas.append({
                "title": item[0],
                "pic": item[1]
            })
            logger.info(f"正在爬取中......图片下载中")
            img_url = requests.get(item[1])
            if img_url.status_code == 200:
                with open(f"{target_file}/{item[0]}.jpg", "wb") as f:
                    f.write(img_url.content)
                    logger.info(f"正在爬取中......图片保存成功")
    with open("doubai.json", "w", encoding="utf-8") as f:
        json.dump(all_datas, f, ensure_ascii=False, indent=4, separators=(',', ': '))

except Exception as e:
    logger.error(f"爬取失败，错误信息是：{e}")



logger.info(f"爬取完成")
if save_to_mongodb(all_datas):
    logger.info(f"数据保存成功")
else:
    logger.info(f"数据保存失败")







# <div class="pic">
#                     <em>14</em>
#                     <a href="https://movie.douban.com/subject/25662329/">
#                         <img width="100" alt="疯狂动物城" src="https://img9.doubanio.com/view/photo/s_ratio_poster/public/p2923817996.jpg">
#                     </a>
#                 </div>